{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 准备好三段文本\n",
    "text1 = \"\"\"\n",
    "篮球，是以手为中心的身体对抗性体育运动，是奥运会核心比赛项目。\n",
    "1891年12月21日，由美国马萨诸塞州斯普林菲尔德基督教青年会训练学校体育教师詹姆士·奈史密斯发明。1896年，篮球运动传入中国天津。1904年，圣路易斯奥运会上第1次进行了篮球表演赛。1936年，篮球在柏林奥运会中被列为正式比赛项目，中国也首次派出篮球队参加奥运会篮球项目。1992年，巴塞罗那奥运会开始，职业选手可以参加奥运会篮球比赛。\n",
    "篮球的最高组织机构为国际篮球联合会，于1932年成立，总部设在瑞士日内瓦。中国最高组织机构为中国篮球协会，于1956年10月成立。\n",
    "\"\"\"\n",
    "\n",
    "text2 = \"\"\"\n",
    "乒乓球，被称为中国的“国球”，是一种世界流行的球类体育项目，包括进攻、对抗和防守。 [1] \n",
    "乒乓球起源于英国，“乒乓球”一名起源自1900年，因其打击时发出“Ping Pong”的声音而得名。在中国大陆以“乒乓球”作为它的官方名称，中国香港及澳门等地区亦同。1926年1月，在德国柏林举行了一次国际乒乓球赛，共有9个国家的64名男运动员参加了比赛。同年12月，国际乒乓球联合会正式成立，并把在伦敦举行的欧洲锦标赛命名为第一届世界乒乓球锦标赛。\n",
    "乒乓球组织机构设有国际乒乓球联合会、亚洲乒乓球联盟、中国乒乓球协会。\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "text3 = \"\"\"\n",
    "羽毛球，是一项隔着球网，使用长柄网状球拍击打用羽毛和软木制作而成的一种小型球类的室内运动项目。羽毛球比赛在长方形的场地上进行，场地中间有网相隔，双方运用各种发球、击球和移动等技战术，将球在网上往返对击，以不使球落在本方有效区域内，或使对方击球失误为胜。 \n",
    "羽毛球运动的起源有很多说法，但最认可的是起源于14—15世纪的日本。而现代羽毛球运动是起源于印度，形成于英国。1875年，羽毛球运动正式出现于人们的视野中。1893年，英国的羽毛球俱乐部逐渐发展起来，成立了第一个羽毛球协会，规定了场地的要求和运动的标准。1939年，国际羽联通过了各会员国共同遵守的第一部《羽毛球规则》。2006年，国际羽毛球联合会（IBF）的正式名称更改为羽毛球世界联合会（BWF），即世界羽联。 \n",
    "羽毛球运动的最高组织机构是世界羽联，1934年在伦敦成立。中国最高组织机构是中国羽毛球协会，1958年9月11日在武汉成立。\n",
    "\"\"\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 调用gensim的TF-IDF模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/xuzhanhong/opt/anaconda3/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora, models, matutils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get text\n",
    "count1, count2, count3 = get_words(text1), get_words(text2), get_words(text3)\n",
    "# 语料库\n",
    "count_list = [count1, count2, count3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# training by TfidfModel in gensim\n",
    "dictionary = corpora.Dictionary(count_list)\n",
    "new_dict = {v:k for k,v in dictionary.token2id.items()}\n",
    "corpus2 = [dictionary.doc2bow(count) for count in count_list]\n",
    "tfidf2 = models.TfidfModel(corpus2)\n",
    "corpus_tfidf = tfidf2[corpus2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training by gensim Tfidf Model.......\n",
      "\n",
      "Top words in document 1\n",
      "    Word: 篮球, TF-IDF: 0.54722\n",
      "    Word: 奥运会, TF-IDF: 0.45601\n",
      "    Word: 比赛项目, TF-IDF: 0.18241\n",
      "Top words in document 2\n",
      "    Word: 乒乓球, TF-IDF: 0.74579\n",
      "    Word: 举行, TF-IDF: 0.16573\n",
      "    Word: 锦标赛, TF-IDF: 0.16573\n",
      "Top words in document 3\n",
      "    Word: 羽毛球, TF-IDF: 0.68137\n",
      "    Word: 运动, TF-IDF: 0.30971\n",
      "    Word: 场地, TF-IDF: 0.18583\n"
     ]
    }
   ],
   "source": [
    "# output\n",
    "print(\"\\nTraining by gensim Tfidf Model.......\\n\")\n",
    "for i, doc in enumerate(corpus_tfidf):\n",
    "    print(\"Top words in document %d\"%(i + 1))\n",
    "    sorted_words = sorted(doc, key=lambda x: x[1], reverse=True)    #type=list\n",
    "    for num, score in sorted_words[:3]:\n",
    "        print(\"    Word: %s, TF-IDF: %s\"%(new_dict[num], round(score, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 自己实现TF-IDF模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "\n",
    "# 文本预处理\n",
    "def get_words(text):\n",
    "    text = text.strip()\n",
    "    # 分词结果\n",
    "    words = list(jieba.cut(text))\n",
    "    # 中文标点符号\n",
    "    punctuation = r\"\"\"!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~“”？，！【】（）、。：；’‘……￥·\"\"\"\n",
    "    tokens = [w for w in words if w not in punctuation]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['篮球', '是', '以手', '为', '中心', '的', '身体', '对抗性', '体育运动', '是']"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 对text1进行预处理\n",
    "get_words(text1)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "# 统计词频\n",
    "def make_count(text):\n",
    "    words = get_words(text)\n",
    "    filtered = words  # 这里可以增加一个去处停用词的步骤\n",
    "    count = Counter(filtered)  # 计数\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('年', 7),\n",
       " ('篮球', 6),\n",
       " ('奥运会', 5),\n",
       " ('为', 3),\n",
       " ('中国', 3),\n",
       " ('是', 2),\n",
       " ('的', 2),\n",
       " ('比赛项目', 2),\n",
       " ('\\n', 2),\n",
       " ('月', 2),\n",
       " ('参加', 2),\n",
       " ('最高', 2),\n",
       " ('组织', 2),\n",
       " ('机构', 2),\n",
       " ('于', 2),\n",
       " ('成立', 2),\n",
       " ('以手', 1),\n",
       " ('中心', 1),\n",
       " ('身体', 1),\n",
       " ('对抗性', 1),\n",
       " ('体育运动', 1),\n",
       " ('核心', 1),\n",
       " ('1891', 1),\n",
       " ('12', 1),\n",
       " ('21', 1),\n",
       " ('日', 1),\n",
       " ('由', 1),\n",
       " ('美国', 1),\n",
       " ('马萨诸塞州', 1),\n",
       " ('斯普林菲尔德', 1),\n",
       " ('基督教', 1),\n",
       " ('青年会', 1),\n",
       " ('训练', 1),\n",
       " ('学校', 1),\n",
       " ('体育教师', 1),\n",
       " ('詹姆士', 1),\n",
       " ('奈', 1),\n",
       " ('史密斯', 1),\n",
       " ('发明', 1),\n",
       " ('1896', 1),\n",
       " ('篮球运动', 1),\n",
       " ('传入', 1),\n",
       " ('天津', 1),\n",
       " ('1904', 1),\n",
       " ('圣路易斯', 1),\n",
       " ('上', 1),\n",
       " ('第', 1),\n",
       " ('1', 1),\n",
       " ('次', 1),\n",
       " ('进行', 1),\n",
       " ('了', 1),\n",
       " ('表演赛', 1),\n",
       " ('1936', 1),\n",
       " ('在', 1),\n",
       " ('柏林', 1),\n",
       " ('中', 1),\n",
       " ('被', 1),\n",
       " ('列为', 1),\n",
       " ('正式', 1),\n",
       " ('也', 1),\n",
       " ('首次', 1),\n",
       " ('派出', 1),\n",
       " ('篮球队', 1),\n",
       " ('项目', 1),\n",
       " ('1992', 1),\n",
       " ('巴塞罗那奥运会', 1),\n",
       " ('开始', 1),\n",
       " ('职业', 1),\n",
       " ('选手', 1),\n",
       " ('可以', 1),\n",
       " ('篮球比赛', 1),\n",
       " ('国际', 1),\n",
       " ('联合会', 1),\n",
       " ('1932', 1),\n",
       " ('总部', 1),\n",
       " ('设在', 1),\n",
       " ('瑞士', 1),\n",
       " ('日内瓦', 1),\n",
       " ('中国篮球', 1),\n",
       " ('协会', 1),\n",
       " ('1956', 1),\n",
       " ('10', 1)]"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 对文本1进行词频统计，同时按照词频进行排序\n",
    "make_count(text1).most_common()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tf(word, count):\n",
    "    \"\"\"\n",
    "    计算词频\n",
    "    \n",
    "    Args:\n",
    "        word (str): [要计算tf的单词]\n",
    "        count (Counter): [当前文章中每个单词及对应词频组成的字典类型数据结构]\n",
    "    \"\"\"\n",
    "    return count[word] / sum(count.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对语料库中所有文本进行词频统计\n",
    "count1, count2, count3 = make_count(text1), make_count(text2), make_count(text3)\n",
    "count_list = [count1, count2, count3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def n_containing(word, count_list):\n",
    "    \"\"\"\n",
    "    计算所有文章中有多少篇文章包含word\n",
    "\n",
    "    Args:\n",
    "        word (str): [指定单词]\n",
    "        count_list (list): [由所有文章的Counter组成的list]\n",
    "\n",
    "    Returns:\n",
    "        int: [包含word的文章篇数]\n",
    "    \"\"\"\n",
    "    return sum(1 for count in count_list if word in count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_containing('篮球', count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def idf(word, count_list):\n",
    "    \"\"\"\n",
    "    计算逆文档频率\n",
    "\n",
    "    Args:\n",
    "        word (str): [要计算idf的单词]\n",
    "        count_list (list): [所有文章的count组成的list]\n",
    "\n",
    "    Returns:\n",
    "        float: [指定单词的idf值]\n",
    "    \"\"\"\n",
    "    return math.log2(len(count_list) / (n_containing(word, count_list)))    # 以2为底的对数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.584962500721156"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf('篮球', count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfidf(word, count, count_list):\n",
    "    \"\"\"\n",
    "    Calculate TF-IDF\n",
    "\n",
    "    Args:\n",
    "        word (str): [要计算tfidf的单词]\n",
    "        count (Counter): [当前文章中每个单词及对应词频组成的字典类型数据结构]\n",
    "        count_list (list): [所有文章的count组成的list]\n",
    "\n",
    "    Returns:\n",
    "        [float]: [指定单词word的tfidf值]\n",
    "    \"\"\"\n",
    "    return tf(word, count) * idf(word, count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.08490870539577622"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf(\"篮球\", count1, count_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training by original algorithm......\n",
      "\n",
      "Top words in document 1\n",
      "    Word: 篮球, TF-IDF: 0.08491\n",
      "    Word: 奥运会, TF-IDF: 0.07076\n",
      "    Word: 比赛项目, TF-IDF: 0.0283\n",
      "Top words in document 2\n",
      "    Word: 乒乓球, TF-IDF: 0.12089\n",
      "    Word: 举行, TF-IDF: 0.02686\n",
      "    Word: 锦标赛, TF-IDF: 0.02686\n",
      "Top words in document 3\n",
      "    Word: 羽毛球, TF-IDF: 0.09033\n",
      "    Word: 运动, TF-IDF: 0.04106\n",
      "    Word: 场地, TF-IDF: 0.02464\n"
     ]
    }
   ],
   "source": [
    "print(\"Training by original algorithm......\\n\")\n",
    "for i, count in enumerate(count_list):\n",
    "    print(\"Top words in document %d\"%(i + 1))\n",
    "    scores = {word: tfidf(word, count, count_list) for word in count}\n",
    "    sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)    #type=list\n",
    "\n",
    "    for word, score in sorted_words[:3]:\n",
    "        print(\"    Word: %s, TF-IDF: %s\"%(word, round(score, 5)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def unitvec(sorted_words):\n",
    "    \"\"\" 对向量做规范化，normalize \"\"\"\n",
    "    lst = [item[1] for item in sorted_words]\n",
    "    L2Norm = math.sqrt(sum(np.array(lst)*np.array(lst)))\n",
    "    unit_vector = [(item[0], item[1]/L2Norm) for item in sorted_words]\n",
    "    return unit_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training by original algorithm......\n",
      "\n",
      "Top words in document 1\n",
      "    Word: 篮球, TF-IDF: 0.54722\n",
      "    Word: 奥运会, TF-IDF: 0.45601\n",
      "    Word: 比赛项目, TF-IDF: 0.18241\n",
      "Top words in document 2\n",
      "    Word: 乒乓球, TF-IDF: 0.74579\n",
      "    Word: 举行, TF-IDF: 0.16573\n",
      "    Word: 锦标赛, TF-IDF: 0.16573\n",
      "Top words in document 3\n",
      "    Word: 羽毛球, TF-IDF: 0.68137\n",
      "    Word: 运动, TF-IDF: 0.30971\n",
      "    Word: 场地, TF-IDF: 0.18583\n"
     ]
    }
   ],
   "source": [
    "print(\"Training by original algorithm......\\n\")\n",
    "for i, count in enumerate(count_list):\n",
    "    print(\"Top words in document %d\"%(i + 1))\n",
    "    scores = {word: tfidf(word, count, count_list) for word in count}\n",
    "    sorted_words = sorted(scores.items(), key=lambda x: x[1], reverse=True)    # type=list\n",
    "    sorted_words = unitvec(sorted_words)\n",
    "    for word, score in sorted_words[:3]:\n",
    "        print(\"    Word: %s, TF-IDF: %s\"%(word, round(score, 5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "a03dfaf43c47ce4d1c68c370fb469d26094a23289322938b5ed8f9ca85d6677e"
  },
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
